# -*- coding: utf-8 -*-
import sys,os
from urllib import *
from httplib import *
import BeautifulSoup as Bs
from apps.main.models.entree import Entree

class CollectWebInfo:
    
    HEADERS = {"Content-type": "application/x-www-form-urlencoded",  
                        'Accept-Language':'zh-cn',  
                        'User-Agent': 'Mozilla/4.0 (compatible; MSIE 6.0;Windows NT 5.0)',  
                        "Accept": "text/plain"}  
    GET='GET'
    POST='POST'
    START_STR = '<div class="entry fix">'
    END_STR = '<div'
    def __init__(self, host):
        self.host = host
        self.conn = HTTPConnection(host) 
        
        #self.entrees = {}
    def get_response(self, url):
        self.conn.request(self.GET, url)
        response = self.conn.getresponse()
        return response

    def get_article_links(self, response):
        soup = Bs.BeautifulSoup(response.read())
        return soup.findAll('a', {"class":"entry-title", "rel":"bookmark"})
    
    def extract_main_content(self, links):
        entree = Entree()
        for link in links:
            response = self.get_response(link['href'])
            temp_entree = self.get_main_area(response)
          #  print temp_entree
            entree.add_entree(title=link['title'], link=link['href'], content=temp_entree)
            
    
    def get_main_area(self, response):
        return self.wrap_part(response.read())
    
    def wrap_part(self, data):
        start = data.find(self.START_STR)
        start += len(self.START_STR)
        end = data.find(self.END_STR, start)
        return data[start:end]
    
    
def main():
    webinfo=CollectWebInfo('www.36kr.com')
    links = webinfo.get_links(webinfo.get_response('/'))
    for link in links:
        print link
def main2():
    webinfo=CollectWebInfo('www.36kr.com')
    links = webinfo.get_article_links(webinfo.get_response('/'))
    webinfo.extract_main_content(links)
  #  print webinfo.entrees.items()
  #  print webinfo.entrees.keys()
  #  print webinfo.entrees.values()[0]
if __name__ == '__main__':
    main2()
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    